/*
 * TOP SECRET Copyright 2006-2015 Transsion.com All right reserved. This software is the confidential and proprietary
 * information of Transsion.com ("Confidential Information"). You shall not disclose such Confidential Information and
 * shall use it only in accordance with the terms of the license agreement you entered into with Transsion.com.
 */
package com.yunji.framework_template.web.service.crawler;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.transaction.Transactional;

import org.apache.commons.httpclient.HttpException;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Tag;
import org.jsoup.select.Elements;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;

import com.alibaba.fastjson.JSON;

import com.yunji.framework_template.BaseTestCase;
import com.yunji.framework_template.common.http.HttpClientUtil;
import com.yunji.framework_template.common.web.HttpResult;
import com.yunji.framework_template.orm.persistence.mapper.SitesInfoMapper;
import com.yunji.framework_template.orm.persistence.mapper.SitesTypeMapper;
import com.yunji.framework_template.orm.persistence.model.SitesInfo;
import com.yunji.framework_template.orm.persistence.model.SitesType;

/**
 * ClassName:BestIndiaSitesCrawler <br/>
 * Date: 2018年11月14日 下午5:34:02 <br/>
 * 
 * @author fenglibin1982@163.com
 * @Blog http://blog.csdn.net/fenglibing
 * @version
 * @see
 */
@Ignore
@Transactional
public class BestIndiaSitesCrawler extends BaseTestCase {

    private static final Map<String, String>  header        = new HashMap<String, String>();
    private static final Map<String, Integer> types         = new HashMap<String, Integer>();
    private List<SitesType>                   sitesTypeList = null;

    @Autowired
    private SitesTypeMapper                   sitesTypeMapper;
    @Autowired
    private SitesInfoMapper                   sitesInfoMapper;

    @Before
    public void before() {
        header.put("Accept", "application/json, text/javascript, */*; q=0.01");
        header.put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
        header.put("Origin", "http://www.bestindiansites.com");
        header.put("Refere", "http://www.bestindiansites.com/category/business/");
        header.put("User-Agent",
                   "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/70.0.3538.77 Chrome/70.0.3538.77 Safari/537.36");
        header.put("X-Requested-With", "XMLHttpRequest");
        sitesTypeList = sitesTypeMapper.selectAll();
        for (SitesType sitesType : sitesTypeList) {
            types.put(sitesType.getCode(), sitesType.getId());
        }
        // 将列表中的分类按code的长短进行分类
        sitesTypeList.sort(new Comparator<SitesType>() {

            @Override
            public int compare(SitesType o1, SitesType o2) {
                return o2.getCode().length() - o1.getCode().length();
            }
        });
    }

    @Test
    public void crawl() throws HttpException, IOException {

        String url = "http://www.bestindiansites.com/2014/wp-content/themes/flavor/functions/ajax.php";
        Map<String, String> param = new HashMap<String, String>();
        param.put("sorter", "recent");
        param.put("location", "");
        param.put("loop", "main loop");
        param.put("action", "sort");
        param.put("view", "grid");
        param.put("columns", "3");
        param.put("paginated", "1");
        param.put("currentquery[category_name]", "business");
        String bigTypesInfo = "BUSINESS,business,2\n" + "ECOMMERCE & RETAIL,ecommerce-retail,1\n"
                              + "EDUCATION,education,2\n" + "ENTERTAINMENT,entertainment,2\n" + "HEALTH,health,1\n"
                              + "HOTEL,hotel,1\n" + "INTERNET & SOFTWARE,internet-software,1\n"
                              + "LIFESTYLE,lifestyle,3\n" + "POPULAR,popular,3\n" + "TECHNOLOGY,technology,1\n"
                              + "TRAVEL,travel,1";
        String[] bigTypes = bigTypesInfo.split("\n");
        Map<String, String> allTypesUrls = new HashMap<String, String>();
        for (String bigType : bigTypes) {
            String[] typeArr = bigType.split(",");
            String typeName = typeArr[0];
            String urlTypeName = typeArr[1];
            param.put("currentquery[category_name]", urlTypeName);
            int pages = Integer.parseInt(typeArr[2]);
            Map<String, String> urlMap = new HashMap<String, String>();
            for (int page = 1; page <= pages; page++) {
                param.put("paginated", String.valueOf(page));
                HttpResult result = HttpClientUtil.doPost(url, param, header);
                Element contentElement = new Element(Tag.valueOf("body"), "");
                contentElement.html(JSON.parseObject(result.getContent()).getString("content"));
                Elements urlEles = contentElement.select("a");
                for (Element element : urlEles) {
                    String subUrl = element.absUrl("href");
                    String title = element.text();
                    if (subUrl.length() == 0 || title.length() == 0) {
                        continue;
                    }
                    urlMap.put(title, subUrl);
                }
            }
            for (Map.Entry<String, String> entry : urlMap.entrySet()) {
                String title = entry.getKey();
                String subUrl = entry.getValue();
                allTypesUrls.put(title, subUrl);
            }
        }
        for (Map.Entry<String, String> entry : allTypesUrls.entrySet()) {
            SitesType siteType = getSitesType(entry.getKey());
            List<SiteVo> siteVoList = getSitesInSinglePage(entry.getValue());
            if (siteVoList == null || siteVoList.size() == 0) {
                System.out.println(entry.getKey() + "," + siteType.getCode() + ","
                                   + entry.getValue());
                continue;
            }
            for (SiteVo vo : siteVoList) {
                SitesInfo siteInfo = new SitesInfo();
                siteInfo.setActive(true);
                siteInfo.setAdded(new Date());
                siteInfo.setDescription(vo.getDesc());
                siteInfo.setName(vo.getName());
                siteInfo.setScreenshot(vo.getScreenShot());
                siteInfo.setTitle(vo.getName());
                siteInfo.setTypeId(siteType.getId());
                siteInfo.setUrl(vo.getUrl());
                sitesInfoMapper.insert(siteInfo);
            }
        }

    }

    // 根据标题获取其分类
    private SitesType getSitesType(String title) {
        for (SitesType sitesType : sitesTypeList) {
            if (title.toLowerCase().indexOf(sitesType.getCode().toLowerCase()) >= 0) {
                return sitesType;
            }
        }
        for (SitesType sitesType : sitesTypeList) {
            String typeCode = sitesType.getCode();
            if (typeCode.indexOf(" ") > 0) {
                typeCode = typeCode.split(" ")[0].trim();// 第一个词做为关键词
                if (title.toLowerCase().indexOf(sitesType.getCode().toLowerCase()) >= 0) {
                    return sitesType;
                }
            }
        }
        return new SitesType();
    }

    private List<SiteVo> getSitesInSinglePage(String url) {
        try {
            Connection conn = Jsoup.connect(url);
            conn.timeout(20000);
            Document doc = conn.get();
            Element contentElement = doc.select("ol").first();
            Elements elements = contentElement.children();
            List<SiteVo> siteList = new ArrayList<SiteVo>();
            SiteVo siteVo = new SiteVo();
            int index = 1;
            for (Element subEle : elements) {
                if (index % 2 == 0) {
                    String href = null;
                    String imgUrl = null;
                    Elements aElement = subEle.select("a");
                    if (aElement != null && aElement.first() != null) {
                        href = aElement.first().absUrl("href");
                        imgUrl = aElement.first().select("img").first().absUrl("src");
                        siteVo.setScreenShot(imgUrl);
                        siteVo.setUrl(href);
                    } else {
                        continue;
                    }

                } else {
                    String text = subEle.text();
                    if (index % 3 == 0) {
                        siteVo.setDesc(text);
                        siteList.add(siteVo);
                        siteVo = new SiteVo();
                        index = 0;
                    } else if (index % 1 == 0) {
                        siteVo.setName(text);
                    }
                }
                index++;
            }
            return siteList;
        } catch (Exception e) {
            return null;
        }
    }

}
